<?php
// +----------------------------------------------------------------------
// | ThinkCMF [ WE CAN DO IT MORE SIMPLE ]
// +----------------------------------------------------------------------
// | Copyright (c) 2013-2018 http://www.thinkcmf.com All rights reserved.
// +----------------------------------------------------------------------
// | Licensed ( http://www.apache.org/licenses/LICENSE-2.0 )
// +----------------------------------------------------------------------
// | Author: guosheng1987@126.com 
// +----------------------------------------------------------------------
namespace plugins\crawler\service; 

use GuzzleHttp\Client;
use GuzzleHttp\Psr7\Request;
use GuzzleHttp\Pool;
use GuzzleHttp\Cookie\CookieJar;
use phpQuery;

//use QL\QueryList;

use plugins\crawler\model\CrawlerSiteModel;
use plugins\crawler\model\CrawlerUrlModel;
use plugins\crawler\model\CrawlerRuleModel;
use plugins\crawler\model\CrawlerCollectionModel;

use app\portal\model\PortalPostModel;
use app\portal\model\PortalCategoryModel;

use think\Db;

class CrawlerFenxiangdashiService extends CrawlerService
{   
    //GuzzleHttp Client 
    public $client;
    //http headers
    public $headers;

    public $crawlerInstance;

    /**
     * 构造函数
     *
     */
    public function __construct()
    {
        //import('phpQuery.phpQuery', EXTEND_PATH);
        
        $this->setClient();

        $this->setHeaders();    
    }   

    /**
     * 设置请求类客户端
     * @access public
     * @return void
     */
    public function setClient()
    {
        $this->client = new Client([
            // Base URI is used with relative requests
            'base_uri' =>  'http://www.fenxiangdashi.com',
            // You can set any number of default request options.
            'timeout'  => 8.0,
            'verify'   => false,
            'delay'    => 3000,
        ]);
        
    }

    /**
     * 设置请求头信息
     * @access public
     * @return void
     */
    public function setHeaders()
    {
        $headers = [
            'User-Agent'      => 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
            'Accept'          => 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
            'Accept-Encoding' => 'gzip, deflate',
            'Accept-Language' => 'zh-CN,zh;q=0.9',
            'Cache-Control'   => 'max-age=0',
            'Connection'      => 'keep-alive',
            'Host'            => 'www.fenxiangdashi.com',
            'Referer'         => 'http://www.fenxiangdashi.com', 
        ];
        
        $cookies = [
            'Hm_lvt_e4bbcc19716cbb012b89dd4b137ce342='. time(),
            'Hm_lpvt_e4bbcc19716cbb012b89dd4b137ce342='. time(),
        ];
        $headers['Cookie'] = implode(' ', $cookies);

        $this->headers = $headers;

    }

    /**
     * 通过列表页面html 匹配所有的文章url
     * @access public
     * @param  String $contents 
     * @return Array $collectionPaths
     */
    public function getCollectionPathsByPageHtml($contents)
    {
        $collectionPaths = [];

        if($contents)
        {
            phpQuery::newDocumentHTML($contents);
            //$ql = pq(null);
            //$ql = QueryList::html($contents);
            
            $collectionPaths = pq('ul.articleCon.pt20 a.more')->map(function ($item) {
                return pq($item)->attr('href');                        
            })->get();

            phpQuery::$documents = null;
        
            //$collectionPaths = $qlData->all();
        }

        return $collectionPaths;
    }
    
    /**
     * 通过文章页面html 匹配需要的内容串
     * @access public
     * @param  String $contents 
     * @return String $collectionContent
     */
    public function getCollectionContentsByPageHtml($contents)
    {
        $collectionContents = '';

        if($contents)
        {
            //$ql = QueryList::html($contents);
            
            phpQuery::newDocumentHTML($contents);
            
            $collectionHtml = pq('article.article-content')->html();
           
            phpQuery::$documents = null;

            $matchtimes = preg_match("|<br>账号([\s\S]*)<br>|", $collectionHtml, $matches);

            if($matchtimes)
            {
                $collectionContents = $matches[0];
            }
        }

        return $collectionContents;
    }

    /**
     * 通过采集id 和网站id 获取文章id
     * @access public
     * @param  Int $collectionId 
     * @param  CrawlerUrlModel $crawlerUrl 
     * @return void
     */
    public function deletePortal($collectionId, $crawlerUrl)
    {
        $postId = Db::name('crawler_collection')
            ->where(['crawler_site_id' => $crawlerUrl->crawler_site_id, 'collection_id' => $collectionId])
            ->value('post_id');

        if($postId)
        {
            Db::name('portal_post')
                ->where(['id' => ['in', $postId]])
                ->delete();
            Db::name('portal_category_post')
                ->where(['post_id' => ['in', $postId]])
                ->delete();
            Db::name('portal_tag_post')
                ->where(['post_id' => ['in', $postId]])
                ->delete();
        }
        
    }

    /**
     * 生成文章标题
     * @access private
     * @param  String $categoryName 栏目名
     * @param  String $tails 长尾词
     * @return void
     */
    private function makePortalTitle($categoryName, $tails)
    {
        $title = $categoryName.'账号'.date('Y').'年'.date('m').'月'.date('d').'日更新  ';

        $tailsArr = explode(',', $tails);

        $randKeys = !empty($tailsArr)? array_rand($tailsArr, 2): [];
        
        if(!empty($randKeys))
        {
            $title .= $tailsArr[$randKeys[0]].' '.$tailsArr[$randKeys[1]];
        }

        return $title;
    }

    /**
     * 生成文章摘要
     * @access private
     * @param  String $title 文章标题
     * @param  String $collectionContents 采集文本
     * @return void
     */
    private function makePortalExcerpt($title, $collectionContents)
    {
        $excerpt = mb_substr(strip_tags($title.'  '.$collectionContents),0,300);

        return $excerpt;
    } 

    /**
     * 生成文章内容
     * @access private
     * @param  String $title 文章标题
     * @param  String $collectionContents 采集文本
     * @param  String $categoryName 
     * @return void
     */
    private function makePortalContent($title, $collectionContents, $categoryName)
    {
        $contentBottom = '分享喵(www.fenxiangmiao.com)，将持续不断的为网友们分享'.$categoryName.'账号资源，请分享给你们的朋友们哦。免费账号由于使用人数过多，5分钟内可能失效，如需稳定会员加微信<b>machen77</b>，官网半价秒发货<br>';
        
        $theme = cmf_get_current_theme();

        $contentImg = '<img src="/themes/'.$theme.'/public/assets/images/qrcode.jpg" style="box-sizing: border-box; border: 0px; vertical-align: middle; max-width: 100%; height: auto !important;">';
        
        $content = $title.$collectionContents.$contentBottom.$contentImg;
        
        return $content;
    } 


    /**
     * 保存文章和和相关数据
     * @access public
     * @param  Int $categoryId 
     * @param  String $title  
     * @param  String $excerpt 
     * @param  String $content 
     * @return void|int
     */
    public function addPortalAndTag($categoryId, $title, $excerpt ,$content)
    {
        $data = [
            'user_id'       => '1',
            'categories'    => $categoryId,
            'post_title'    => $title,
            'post_keywords' => '',
            'post_source'   => '',
            'post_excerpt'  => $excerpt,
            'more' => [
                'thumbnail' => '',
                'template'  => ''
            ],
            'published_time' => date('Y-m-d H:i:s'),
            'post_status'    => '1',
            'post_content'   => $content,
        ];

        $portalPostModel = new PortalPostModel();
        
        $portalPostModel->allowField(true)->data($data, true)->isUpdate(false)->save();
        
        $categories = $data['categories'];

        if ($categories) {
            $categories = explode(',', $categories);
        }

        $portalPostModel->categories()->save($categories);

        $data['post_keywords'] = str_replace('，', ',', $data['post_keywords']);

        $keywords = explode(',', $data['post_keywords']);

        $portalPostModel->addTags($keywords, $portalPostModel->id);

        return $portalPostModel->id;
    }

    /**
     * 采集结果 插入采集数据表 
     * @access public
     * @param  Int $collectionId 
     * @param  Int $postId 
     * @param  CrawlerUrl $crawlerUrl 
     * @param  String $title 
     * @param  String $collectionContents 
     * @return void
     */
    public function addCrawlerCollection($collectionId, $postId, $crawlerUrl, $title, $collectionContents)
    {
        $time = time();

        $data = [];

        $data['crawler_site_id'] = $crawlerUrl->crawler_site_id;
        $data['crawler_url_id']  = $crawlerUrl->id;
        $data['crawler_rule_id'] = $crawlerUrl->crawler_rule_id;
        $data['collection_id']   = $collectionId;
        $data['post_id']         = $postId;
        $data['title']           = $title;
        $data['content']         = $collectionContents;
        $data['create_time']     = $time;

        $postId = Db::name('crawler_collection')
            ->where(['crawler_site_id' => $data['crawler_site_id'], 'collection_id' => $collectionId])
            ->delete();

        Db::name('crawler_collection')->insert($data);        
    }

    /**
     * 根据栏目id获取文章栏目名
     * @access public
     * @param  Int $categoryId 
     * @return String $categoryName
     */
    public function getPortalCategoryNameById($categoryId)
    {
        $portalCategoryModel = new PortalCategoryModel();
        $categoryName = $portalCategoryModel->find($categoryId)->name;
        
        return $categoryName;
    }

    /**
     * 采集结果 插入采集数据表 
     * @access public
     * @param  Int $collectionId 
     * @param  String $collectionContents 
     * @param  CrawlerUrl $crawlerUrl 
     * @return void
     */
    public function saveCrawlerData($collectionId, $collectionContents, $crawlerUrl)
    {
        $this->deletePortal($collectionId, $crawlerUrl);
        
        $categoryId = $crawlerUrl->crlrule->category_id;
        $tails = $crawlerUrl->crlrule->tails;
        
        $categoryName = $this->getPortalCategoryNameById($categoryId);

        $title = $this->makePortalTitle($categoryName, $tails);
        $excerpt = $this->makePortalExcerpt($title, $collectionContents);        
        $content = $this->makePortalContent($title, $collectionContents, $categoryName);

        $postId = $this->addPortalAndTag($categoryId, $title, $excerpt ,$content);

        $this->addCrawlerCollection($collectionId, $postId, $crawlerUrl, $title, $collectionContents);
    }
    

    /**
     * 开始采集任务
     * 
     * @return array
     */
    public function start()
    {
        $crawlerUrls = CrawlerUrlModel::where('crawler_site_id', '1')->select();
        
        foreach($crawlerUrls as $crawlerUrl)
        {   
            $response = $this->client->request('GET', $crawlerUrl->listurl, [
                'allow_redirects' => false,
                'headers'         => $this->headers,
            ]);
            
            $contents = $response->getBody()->getContents();
            
            $collectionPaths = $this->getCollectionPathsByPageHtml($contents);
            
            $collectionPaths = array_slice($collectionPaths, 0, 4);

            $options['baseuri'] = $crawlerUrl->crlrule->baseuri;
            $options['headers'] = $this->headers;
            
            $requests = function ($collectionPaths) use ($options) {
                foreach ($collectionPaths as $path) {                                                
                    yield new Request('GET', $options['baseuri'] . $path, $options['headers']);                                
                }
            };

            $pool = new Pool($this->client, $requests($collectionPaths), [
                'concurrency' => 3,
                'fulfilled'   => function ($response, $index) use ($collectionPaths, $crawlerUrl) {
                    
                    $contents = $response->getBody()->getContents();

                    $collectionContents = $this->getCollectionContentsByPageHtml($contents);
                    
                    $matchtimes = preg_match('/\d+/', $collectionPaths[$index], $matches);
                    
                    $collectionId = $matches[0];

                    $this->saveCrawlerData($collectionId, $collectionContents, $crawlerUrl);
                
                    echo "page url ".$collectionPaths[$index] .' collect data success!'. PHP_EOL;         
                },
                'rejected'   => function ($reason, $index) use ($collectionPaths) {
                    // this is delivered each failed request
                    echo "page url ".$collectionPaths[$index].' request fail!'. PHP_EOL;
                },
            ]);
    
            // Initiate the transfers and create a promise
            $promise = $pool->promise();
    
            // Force the pool of requests to complete.
            $promise->wait();
            
            echo "list url ".$crawlerUrl->listurl.' request success!'. PHP_EOL;
            
        }
        $this->output->writeln("Command executed success!");
        
    }
    
}